{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8f1d187e-4ef1-4265-9389-7848ac699730",
   "metadata": {},
   "source": [
    "# GPT-3 Performance on CSQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "efe07395-4ece-4591-be00-22afeb6230cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "from pprint import pprint\n",
    "from tenacity import retry, stop_after_attempt, wait_chain, wait_fixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "080d63f4-a5d8-41be-a1d6-2a246bf03baa",
   "metadata": {},
   "outputs": [],
   "source": [
    "@retry(wait=wait_chain(*[wait_fixed(3) for i in range(3)] +\n",
    "                       [wait_fixed(5) for i in range(2)] +\n",
    "                       [wait_fixed(10)]))\n",
    "def completion_with_backoff(**kwargs):\n",
    "    return openai.Completion.create(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a1aa06b2-b4fb-46ea-a736-70ddb73f3354",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f0212001-ae07-4bd4-b205-9a79065320cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_jsonl(path):\n",
    "    data=[]\n",
    "    with open(path, 'r', encoding='utf-8') as reader:\n",
    "        for line in reader:\n",
    "            data.append(json.loads(line))\n",
    "    return data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa1037f4-7926-4653-b5e3-5918671e5305",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = load_jsonl('data/train_rand_split.jsonl')\n",
    "dev_data = load_jsonl('data/dev_rand_split.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "741f2f6a-e4ff-4ec6-959e-c840d3e4080f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1221"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "850533c4-624c-4240-9eb8-24edb73c6d01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answerKey': 'A',\n",
       " 'id': '075e483d21c29a511267ef62bedc0461',\n",
       " 'question': {'question_concept': 'punishing',\n",
       "  'choices': [{'label': 'A', 'text': 'ignore'},\n",
       "   {'label': 'B', 'text': 'enforce'},\n",
       "   {'label': 'C', 'text': 'authoritarian'},\n",
       "   {'label': 'D', 'text': 'yell at'},\n",
       "   {'label': 'E', 'text': 'avoid'}],\n",
       "  'stem': 'The sanctions against the school were a punishing blow, and they seemed to what the efforts the school had made to change?'}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6c5327a7-4472-41cc-8f0e-7a842e4fabc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answerKey': 'B',\n",
       " 'id': '1a7f5b3c65364d9be002576660c914fe',\n",
       " 'question': {'question_concept': 'grape',\n",
       "  'choices': [{'label': 'A', 'text': 'mouth'},\n",
       "   {'label': 'B', 'text': 'grocery cart'},\n",
       "   {'label': 'C', 'text': 'super market'},\n",
       "   {'label': 'D', 'text': 'fruit basket'},\n",
       "   {'label': 'E', 'text': 'fruit market'}],\n",
       "  'stem': 'Where do you put your grapes just before checking out?'}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e0cd630b-f193-4492-b8de-99fe1c13777e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answerKey': 'A',\n",
       " 'id': '1afa02df02c908a558b4036e80242fac',\n",
       " 'question': {'question_concept': 'revolving door',\n",
       "  'choices': [{'label': 'A', 'text': 'bank'},\n",
       "   {'label': 'B', 'text': 'library'},\n",
       "   {'label': 'C', 'text': 'department store'},\n",
       "   {'label': 'D', 'text': 'mall'},\n",
       "   {'label': 'E', 'text': 'new york'}],\n",
       "  'stem': 'A revolving door is convenient for two direction travel, but it also serves as a security measure at a what?'}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "642be16d-842f-4b95-b72f-48ec9907f0a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dc9e714e-1f4c-4610-9954-571531fc06e3",
   "metadata": {},
   "source": [
    "# Find Complex Training Cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a1dac329-6348-4f47-b73e-6058b887a621",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9741"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c5e0cd71-78d4-4694-b5b2-c88da73001df",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_len = list(len(d['question']['stem']) for d in train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9f3a25d6-7b16-480c-8a06-9f74bda31970",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5689,   60, 7884, ..., 5419, 3564, 6458])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argsort(train_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "60b2387c-3d28-475c-ac2e-894c3d70f2d9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What is a hand?\n",
      "Answer Choices\n",
      "(a) body part\n",
      "(b) foot\n",
      "(c) handglide\n",
      "(d) feet\n",
      "(e) help\n",
      "So the answer is (a)\n",
      "\n",
      "Q: What do drugs do?\n",
      "Answer Choices\n",
      "(a) damage\n",
      "(b) cloud mind\n",
      "(c) lower i.q\n",
      "(d) cause illness\n",
      "(e) cause accidents\n",
      "So the answer is (e)\n",
      "\n",
      "Q: What bounds a book?\n",
      "Answer Choices\n",
      "(a) at least few page\n",
      "(b) knowledge\n",
      "(c) many words\n",
      "(d) cover\n",
      "(e) staples\n",
      "So the answer is (d)\n",
      "\n",
      "Q: What color is milk?\n",
      "Answer Choices\n",
      "(a) fridge\n",
      "(b) white\n",
      "(c) chocolate flavored\n",
      "(d) good for babies\n",
      "(e) golden\n",
      "So the answer is (b)\n",
      "\n",
      "Q: What do animals eat?\n",
      "Answer Choices\n",
      "(a) human\n",
      "(b) cuddly pet\n",
      "(c) plant\n",
      "(d) specific\n",
      "(e) eat cake\n",
      "So the answer is (c)\n",
      "\n",
      "Q: Why does wood float?\n",
      "Answer Choices\n",
      "(a) it is a good swimmer\n",
      "(b) solid\n",
      "(c) gilded\n",
      "(d) less dense than water\n",
      "(e) porous\n",
      "So the answer is (d)\n",
      "\n",
      "Q: Where do cows graze?\n",
      "Answer Choices\n",
      "(a) green field\n",
      "(b) dry fields\n",
      "(c) meat grinder\n",
      "(d) red barn\n",
      "(e) countryside\n",
      "So the answer is (a)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in np.argsort(train_len)[:7]:\n",
    "    print('Q: ' + train_data[i]['question']['stem'])\n",
    "    print('Answer Choices')\n",
    "    for c in train_data[i]['question']['choices']:\n",
    "    # for c in ['A', 'B', 'C', 'D', 'E']:\n",
    "        print('(%c) %s' % (c['label'].lower(), c['text']))\n",
    "    # pprint(train_data[i]['question']['choices'])\n",
    "    print('So the answer is (%c)\\n' % train_data[i]['answerKey'].lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "efe4281c-a159-44bb-b933-7769a54d0470",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: Laura and Bob are a married couple who have begun reaching tentative agreement regarding their  sexual incompatibility.  They love each other and want to remain together, but have decided that they can see other people to fulfill their needs as long as they are honest about it.  Then entire negotiation is being done with a surprising amount of what given the subject matter?\n",
      "Answer Choices\n",
      "(a) calmness\n",
      "(b) compromises\n",
      "(c) fucking\n",
      "(d) anger\n",
      "(e) satisfaction\n",
      "So the answer is (a)\n",
      "\n",
      "Q: At reception desk John asked  to see a person who was helping him make a sale.  Lilly came out immediately and took him back to her space, and talked to him about his needs.  He said that he was willing to lower the price to 80,000 if it would sale faster.  Where might John be?\n",
      "Answer Choices\n",
      "(a) large office\n",
      "(b) at hotel\n",
      "(c) building\n",
      "(d) real estate office\n",
      "(e) store room\n",
      "So the answer is (d)\n",
      "\n",
      "Q: Joe and Jane have begun reaching a tentative agreement about the merger of their two companies.  Their negotiations have become less than professional,  and have begun doing something that is very unprofessional and compromises their objectivity.  What might they be doing?\n",
      "Answer Choices\n",
      "(a) compromises\n",
      "(b) eloping\n",
      "(c) calmness\n",
      "(d) fucking\n",
      "(e) handshake\n",
      "So the answer is (d)\n",
      "\n",
      "Q: The building could accommodate many people.  The entrance hall alone was impressive, being wide enough to admit a hundred shoulder to shoulder.  But the building was owned by a billionaire and used only for his personal entertainment.  How would you describe this place?\n",
      "Answer Choices\n",
      "(a) convention center\n",
      "(b) public building\n",
      "(c) large building\n",
      "(d) school\n",
      "(e) town hall\n",
      "So the answer is (c)\n",
      "\n",
      "Q: Joe found that shopping  was dangerous for him.  Even though he had a great job, or maybe because of it, he found himself doing this a great deal. There were always funds left over, but not enough for his taste.  What was he doing when he was shopping?\n",
      "Answer Choices\n",
      "(a) overspending\n",
      "(b) loss of money\n",
      "(c) run out of money\n",
      "(d) spending money\n",
      "(e) deficit\n",
      "So the answer is (a)\n",
      "\n",
      "Q: James noticed that people who communicate well have fewer problems.  Some people communicate easily with strangers, others need to talk with members of their own groups.  A religion is a type of group.  What do members of a religion have in common?\n",
      "Answer Choices\n",
      "(a) believe in god\n",
      "(b) talk to each other\n",
      "(c) pay bills\n",
      "(d) learn from each other\n",
      "(e) believe in ghosts\n",
      "So the answer is (a)\n",
      "\n",
      "Q: John heard a language that he could not understand. He thought that the door was shut, but he eventually realized that there was no door, and that the light source that was blinding his eyes was very familiar.  He was on his back looking at what?\n",
      "Answer Choices\n",
      "(a) sky\n",
      "(b) lamp\n",
      "(c) hallway\n",
      "(d) dard\n",
      "(e) closed room\n",
      "So the answer is (a)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in np.argsort(train_len)[-7:][::-1]:\n",
    "    print('Q: ' + train_data[i]['question']['stem'])\n",
    "    print('Answer Choices')\n",
    "    for c in train_data[i]['question']['choices']:\n",
    "    # for c in ['A', 'B', 'C', 'D', 'E']:\n",
    "        print('(%c) %s' % (c['label'].lower(), c['text']))\n",
    "    # pprint(train_data[i]['question']['choices'])\n",
    "    print('So the answer is (%c)\\n' % train_data[i]['answerKey'].lower())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44156928-195c-48ac-aa5c-73940b381752",
   "metadata": {},
   "source": [
    "# Original Prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f876090e-1e98-4912-a2b0-34d4b5b7d3c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_original = open('lib_prompt/prompt_original.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c4a5d864-248a-421f-a80b-032448226221",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What do people use to absorb extra ink from a fountain pen?\n",
      "Answer Choices:\n",
      "(a) shirt pocket\n",
      "(b) calligrapher’s hand\n",
      "(c) inkwell\n",
      "(d) desk drawer\n",
      "(e) blotter\n",
      "A: The answer must be an item that can absorb ink.\n",
      "Of the above choices, only blotters are used to absorb ink.\n",
      "So the answer is (e).\n",
      "\n",
      "Q: What home entertainment equipment requires cable?\n",
      "Answer Choices:\n",
      "(a) radio shack\n",
      "(b) substation\n",
      "(c) television\n",
      "(d) cabinet\n",
      "A: The answer must require cable.\n",
      "Of the above choices, only television requires cable.\n",
      "So the answer is (c).\n",
      "\n",
      "Q: The fox walked from the city into the forest, what was it looking for?\n",
      "Answer Choices:\n",
      "(a) pretty flowers\n",
      "(b) hen house\n",
      "(c) natural habitat\n",
      "(d) storybook\n",
      "A: The answer must be something in the forest.\n",
      "Of the above choices, only natural habitat is in the forest.\n",
      "So the answer is (b).\n",
      "\n",
      "Q: Sammy wanted to go to where the people were. Where might he go?\n",
      "Answer Choices:\n",
      "(a) populated areas\n",
      "(b) race track\n",
      "(c) desert\n",
      "(d) apartment\n",
      "(e) roadblock\n",
      "A: The answer must be a place with a lot of people.\n",
      "Of the above choices, only populated areas have a lot of people.\n",
      "So the answer is (a).\n",
      "\n",
      "Q: Where do you put your grapes just before checking out?\n",
      "Answer Choices:\n",
      "(a) mouth\n",
      "(b) grocery cart\n",
      "(c) super market \n",
      "(d) fruit basket \n",
      "(e) fruit market\n",
      "A: The answer should be the place where grocery items are placed before checking out.\n",
      "Of the above choices, grocery cart makes the most sense for holding grocery items.\n",
      "So the answer is (b).\n",
      "\n",
      "Q: Google Maps and other highway and street GPS services have replaced what?\n",
      "Answer Choices:\n",
      "(a) united states\n",
      "(b) mexico\n",
      "(c) countryside\n",
      "(d) atlas\n",
      "A: The answer must be something that used to do what Google Maps and GPS services do, which is to give directions.\n",
      "Of the above choices, only atlases are used to give directions.\n",
      "So the answer is (d).\n",
      "\n",
      "Q: Before getting a divorce, what did the wife feel who was doing all the work?\n",
      "Answer Choices:\n",
      "(a) harder\n",
      "(b) anguish\n",
      "(c) bitterness\n",
      "(d) tears\n",
      "(e) sadness\n",
      "A: The answer should be the feeling of someone getting divorced who was doing all the work.\n",
      "Of the above choices, the closest feeling is bitterness.\n",
      "So the answer is (c).\n",
      "   \n"
     ]
    }
   ],
   "source": [
    "print(prompt_original)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "99af177f-d524-4b39-b443-756f45cd6408",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_q = prompt_original + '\\nQ: ' + dev_data[0]['question']['stem'] + '\\n'\n",
    "prompt_q += 'Answer Choices:\\n'\n",
    "for c in train_data[i]['question']['choices']:\n",
    "    prompt_q += '(%c) %s\\n' % (c['label'].lower(), c['text'])\n",
    "prompt_q += 'A:'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "00b19469-1834-403c-b17b-78ed06992c89",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What do people use to absorb extra ink from a fountain pen?\n",
      "Answer Choices:\n",
      "(a) shirt pocket\n",
      "(b) calligrapher’s hand\n",
      "(c) inkwell\n",
      "(d) desk drawer\n",
      "(e) blotter\n",
      "A: The answer must be an item that can absorb ink.\n",
      "Of the above choices, only blotters are used to absorb ink.\n",
      "So the answer is (e).\n",
      "\n",
      "Q: What home entertainment equipment requires cable?\n",
      "Answer Choices:\n",
      "(a) radio shack\n",
      "(b) substation\n",
      "(c) television\n",
      "(d) cabinet\n",
      "A: The answer must require cable.\n",
      "Of the above choices, only television requires cable.\n",
      "So the answer is (c).\n",
      "\n",
      "Q: The fox walked from the city into the forest, what was it looking for?\n",
      "Answer Choices:\n",
      "(a) pretty flowers\n",
      "(b) hen house\n",
      "(c) natural habitat\n",
      "(d) storybook\n",
      "A: The answer must be something in the forest.\n",
      "Of the above choices, only natural habitat is in the forest.\n",
      "So the answer is (b).\n",
      "\n",
      "Q: Sammy wanted to go to where the people were. Where might he go?\n",
      "Answer Choices:\n",
      "(a) populated areas\n",
      "(b) race track\n",
      "(c) desert\n",
      "(d) apartment\n",
      "(e) roadblock\n",
      "A: The answer must be a place with a lot of people.\n",
      "Of the above choices, only populated areas have a lot of people.\n",
      "So the answer is (a).\n",
      "\n",
      "Q: Where do you put your grapes just before checking out?\n",
      "Answer Choices:\n",
      "(a) mouth\n",
      "(b) grocery cart\n",
      "(c) super market \n",
      "(d) fruit basket \n",
      "(e) fruit market\n",
      "A: The answer should be the place where grocery items are placed before checking out.\n",
      "Of the above choices, grocery cart makes the most sense for holding grocery items.\n",
      "So the answer is (b).\n",
      "\n",
      "Q: Google Maps and other highway and street GPS services have replaced what?\n",
      "Answer Choices:\n",
      "(a) united states\n",
      "(b) mexico\n",
      "(c) countryside\n",
      "(d) atlas\n",
      "A: The answer must be something that used to do what Google Maps and GPS services do, which is to give directions.\n",
      "Of the above choices, only atlases are used to give directions.\n",
      "So the answer is (d).\n",
      "\n",
      "Q: Before getting a divorce, what did the wife feel who was doing all the work?\n",
      "Answer Choices:\n",
      "(a) harder\n",
      "(b) anguish\n",
      "(c) bitterness\n",
      "(d) tears\n",
      "(e) sadness\n",
      "A: The answer should be the feeling of someone getting divorced who was doing all the work.\n",
      "Of the above choices, the closest feeling is bitterness.\n",
      "So the answer is (c).\n",
      "   \n",
      "Q: A revolving door is convenient for two direction travel, but it also serves as a security measure at a what?\n",
      "Answer Choices:\n",
      "(a) sky\n",
      "(b) lamp\n",
      "(c) hallway\n",
      "(d) dard\n",
      "(e) closed room\n",
      "A:\n"
     ]
    }
   ],
   "source": [
    "print(prompt_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "11271431-251e-4250-b709-ca392eee190a",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = completion_with_backoff(model=\"text-davinci-002\", \n",
    "                                                prompt=prompt_q, \n",
    "                                                temperature=0, \n",
    "                                                max_tokens=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5c323532-fa2d-4bbc-bf63-05364563dbc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' The answer should be a place where a revolving door would be a security measure.\\nOf the above choices, the only place that makes sense is a closed room.\\nSo the answer is (e).'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response['choices'][0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "523173d9-9c03-4442-9772-b123a8e8ff52",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1221/1221 [42:22<00:00,  2.08s/it]\n"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "total = 0\n",
    "with open('outputs/dev_gpt3_original.txt', 'w') as fd:\n",
    "    for d in tqdm(dev_data):\n",
    "        q = d['question']['stem']\n",
    "        a = d['answerKey'].lower()\n",
    "        \n",
    "        prompt_q = prompt_original + '\\nQ: ' + q + '\\n'\n",
    "        prompt_q += 'Answer Choices:\\n'\n",
    "        for c in d['question']['choices']:\n",
    "            prompt_q += '(%c) %s\\n' % (c['label'].lower(), c['text'])\n",
    "        prompt_q += 'A:'\n",
    "\n",
    "        response = completion_with_backoff(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_q, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "\n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('Q: %s\\nA_model:\\n%s\\nA:\\n%s\\n\\n' % (q, ans_model, a))\n",
    "        \n",
    "        ans_ = ans_model.split('(')[1][0]\n",
    "        if(ans_ == a): acc += 1\n",
    "        total += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d41999ad-a1d8-46a0-9571-a0390f4704ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The answer should be a place where farmland is sold.\n",
      "Of the above choices, farming areas is the most likely place to buy farmland.\n",
      "So the answer is (d).\n"
     ]
    }
   ],
   "source": [
    "print(ans_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "1df8019b-29b0-4078-aa32-f8968bc794c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1221 correct 933 acc 0.76\n"
     ]
    }
   ],
   "source": [
    "print('Total %d correct %d acc %.4f' % (total, acc, acc / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "5247930e-6f88-47eb-a3b6-b7f5b60c83db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7641277641277642"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "933 / 1221"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83c08179-a89b-401c-8cd1-8675bd6929e9",
   "metadata": {},
   "source": [
    "# Simple Prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "ce83fe71-bbb4-44a9-8a69-77953baa5b60",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_simple = open('lib_prompt/prompt_simple.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "47578f19-5497-4c7f-bd3e-faca8330215f",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What is a hand?\n",
      "Answer Choices\n",
      "(a) body part\n",
      "(b) foot\n",
      "(c) handglide\n",
      "(d) feet\n",
      "(e) help\n",
      "A: The answer should be a hypernym of hand.\n",
      "Of the above choices, only body part is a hypernym of hand.\n",
      "So the answer is (a)\n",
      "\n",
      "Q: What do drugs do?\n",
      "Answer Choices\n",
      "(a) damage\n",
      "(b) cloud mind\n",
      "(c) lower i.q\n",
      "(d) cause illness\n",
      "(e) cause accidents\n",
      "A: The answer should be common direct side effects of drugs.\n",
      "Of the above choices, only causing accidents is the common direct effect of drugs. \n",
      "So the answer is (e)\n",
      "\n",
      "Q: What bounds a book?\n",
      "Answer Choices\n",
      "(a) at least few page\n",
      "(b) knowledge\n",
      "(c) many words\n",
      "(d) cover\n",
      "(e) staples\n",
      "A: The answer should be the thing bounding a book.\n",
      "Of the above choices, only cover bounds a book. \n",
      "So the answer is (d)\n",
      "\n",
      "Q: What color is milk?\n",
      "Answer Choices\n",
      "(a) fridge\n",
      "(b) white\n",
      "(c) chocolate flavored\n",
      "(d) good for babies\n",
      "(e) golden\n",
      "A: The answer should be the common color of milk.\n",
      "Of the above choices, white is the common color of milk. \n",
      "So the answer is (b)\n",
      "\n",
      "Q: What do animals eat?\n",
      "Answer Choices\n",
      "(a) human\n",
      "(b) cuddly pet\n",
      "(c) plant\n",
      "(d) specific\n",
      "(e) eat cake\n",
      "A: The answer should be common food of animals. \n",
      "Of the above choices, plant is the common food of animals. \n",
      "So the answer is (c)\n",
      "\n",
      "Q: Why does wood float?\n",
      "Answer Choices\n",
      "(a) it is a good swimmer\n",
      "(b) solid\n",
      "(c) gilded\n",
      "(d) less dense than water\n",
      "(e) porous\n",
      "A: The answer should be related to the dense of the wood. \n",
      "Of the above choices, only d says that wood has less dense than water. \n",
      "So the answer is (d)\n",
      "\n",
      "Q: Where do cows graze?\n",
      "Answer Choices\n",
      "(a) green field\n",
      "(b) dry fields\n",
      "(c) meat grinder\n",
      "(d) red barn\n",
      "(e) countryside\n",
      "A: The answer should be some place that attract cows. \n",
      "Of the above choices, green field attracts cows because cows eat grass. \n",
      "So the answer is (a)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_simple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0dabfd2e-1768-406e-ac43-67c6374fdeaf",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍                          | 969/1221 [34:11<08:53,  2.12s/it]\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [60]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m ans_model \u001b[38;5;241m=\u001b[39m response[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m     20\u001b[0m fd\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mQ: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mA_model:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mA:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m%\u001b[39m (q, ans_model, a))\n\u001b[0;32m---> 22\u001b[0m ans_ \u001b[38;5;241m=\u001b[39m \u001b[43mans_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m(ans_ \u001b[38;5;241m==\u001b[39m a): acc \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m     24\u001b[0m total \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "total = 0\n",
    "with open('outputs/dev_gpt3_simple.txt', 'w') as fd:\n",
    "    for d in tqdm(dev_data):\n",
    "        q = d['question']['stem']\n",
    "        a = d['answerKey'].lower()\n",
    "        \n",
    "        prompt_q = prompt_simple + '\\nQ: ' + q + '\\n'\n",
    "        prompt_q += 'Answer Choices:\\n'\n",
    "        for c in d['question']['choices']:\n",
    "            prompt_q += '(%c) %s\\n' % (c['label'].lower(), c['text'])\n",
    "        prompt_q += 'A:'\n",
    "\n",
    "        response = completion_with_backoff(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_q, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "\n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('Q: %s\\nA_model:\\n%s\\nA:\\n%s\\n\\n' % (q, ans_model, a))\n",
    "        \n",
    "        ans_ = ans_model.split('(')\n",
    "        if(len(ans_) > 1): ans_ = ans_[1][0]\n",
    "        else: ans_ = 'a'\n",
    "        \n",
    "        if(ans_ == a): acc += 1\n",
    "        total += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "14e22a48-e520-4fb4-9f51-5aace052723d",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: What is a hand?\n",
      "Answer Choices\n",
      "(a) body part\n",
      "(b) foot\n",
      "(c) handglide\n",
      "(d) feet\n",
      "(e) help\n",
      "A: The answer should be a hypernym of hand.\n",
      "Of the above choices, only body part is a hypernym of hand.\n",
      "So the answer is (a)\n",
      "\n",
      "Q: What do drugs do?\n",
      "Answer Choices\n",
      "(a) damage\n",
      "(b) cloud mind\n",
      "(c) lower i.q\n",
      "(d) cause illness\n",
      "(e) cause accidents\n",
      "A: The answer should be common direct side effects of drugs.\n",
      "Of the above choices, only causing accidents is the common direct effect of drugs. \n",
      "So the answer is (e)\n",
      "\n",
      "Q: What bounds a book?\n",
      "Answer Choices\n",
      "(a) at least few page\n",
      "(b) knowledge\n",
      "(c) many words\n",
      "(d) cover\n",
      "(e) staples\n",
      "A: The answer should be the thing bounding a book.\n",
      "Of the above choices, only cover bounds a book. \n",
      "So the answer is (d)\n",
      "\n",
      "Q: What color is milk?\n",
      "Answer Choices\n",
      "(a) fridge\n",
      "(b) white\n",
      "(c) chocolate flavored\n",
      "(d) good for babies\n",
      "(e) golden\n",
      "A: The answer should be the common color of milk.\n",
      "Of the above choices, white is the common color of milk. \n",
      "So the answer is (b)\n",
      "\n",
      "Q: What do animals eat?\n",
      "Answer Choices\n",
      "(a) human\n",
      "(b) cuddly pet\n",
      "(c) plant\n",
      "(d) specific\n",
      "(e) eat cake\n",
      "A: The answer should be common food of animals. \n",
      "Of the above choices, plant is the common food of animals. \n",
      "So the answer is (c)\n",
      "\n",
      "Q: Why does wood float?\n",
      "Answer Choices\n",
      "(a) it is a good swimmer\n",
      "(b) solid\n",
      "(c) gilded\n",
      "(d) less dense than water\n",
      "(e) porous\n",
      "A: The answer should be related to the dense of the wood. \n",
      "Of the above choices, only d says that wood has less dense than water. \n",
      "So the answer is (d)\n",
      "\n",
      "Q: Where do cows graze?\n",
      "Answer Choices\n",
      "(a) green field\n",
      "(b) dry fields\n",
      "(c) meat grinder\n",
      "(d) red barn\n",
      "(e) countryside\n",
      "A: The answer should be some place that attract cows. \n",
      "Of the above choices, green field attracts cows because cows eat grass. \n",
      "So the answer is (a)\n",
      "\n",
      "Q: In what Spanish speaking North American country can you get a great cup of coffee?\n",
      "Answer Choices:\n",
      "(a) mildred's coffee shop\n",
      "(b) mexico\n",
      "(c) diner\n",
      "(d) kitchen\n",
      "(e) canteen\n",
      "A:\n"
     ]
    }
   ],
   "source": [
    "print(prompt_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "89b43922-fb86-4de2-bec6-2f29c835f415",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 969 correct 703 acc 0.73\n"
     ]
    }
   ],
   "source": [
    "print('Total %d correct %d acc %.2f' % (total, acc, acc / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "3206c434-f06f-4980-9e7d-26a816805941",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 252/252 [09:02<00:00,  2.15s/it]\n"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "total = 0\n",
    "with open('outputs/dev_gpt3_simple_from_969.txt', 'w') as fd:\n",
    "    for d in tqdm(dev_data[969:]):\n",
    "        q = d['question']['stem']\n",
    "        a = d['answerKey'].lower()\n",
    "        \n",
    "        prompt_q = prompt_simple + '\\nQ: ' + q + '\\n'\n",
    "        prompt_q += 'Answer Choices:\\n'\n",
    "        for c in d['question']['choices']:\n",
    "            prompt_q += '(%c) %s\\n' % (c['label'].lower(), c['text'])\n",
    "        prompt_q += 'A:'\n",
    "\n",
    "        response = completion_with_backoff(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_q, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "\n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('Q: %s\\nA_model:\\n%s\\nA:\\n%s\\n\\n' % (q, ans_model, a))\n",
    "        \n",
    "        ans_ = ans_model.split('(')\n",
    "        if(len(ans_) > 1): ans_ = ans_[1][0]\n",
    "        else: ans_ = 'a'\n",
    "        \n",
    "        if(ans_ == a): acc += 1\n",
    "        total += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "7de1eaea-d003-41f1-aca4-6440a847f720",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 252 correct 172 acc 0.68\n"
     ]
    }
   ],
   "source": [
    "print('Total %d correct %d acc %.2f' % (total, acc, acc / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "7388591e-e05a-4cef-80ae-244c9e453d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7166257166257166"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(172 + 703) / (969 + 252)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e3a40f4-2101-42fe-88fc-9e79f58e17ee",
   "metadata": {},
   "source": [
    "# Complex Prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "74c4a7aa-17c9-4bee-a701-b7c04fbcc62b",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_complex = open('lib_prompt/prompt_complex.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2e6a2e52-59d3-4813-b174-fe20b97187c7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|██████████▌                                                                                                                      | 100/1221 [03:23<40:43,  2.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 100 correct 65 acc 0.6500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█████████████████████▏                                                                                                           | 200/1221 [06:57<32:19,  1.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 200 correct 140 acc 0.7000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|███████████████████████████████▋                                                                                                 | 300/1221 [10:34<29:47,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 300 correct 208 acc 0.6933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|██████████████████████████████████████████▎                                                                                      | 400/1221 [14:12<24:47,  1.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 400 correct 272 acc 0.6800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████████████████████████████▊                                                                            | 500/1221 [17:52<30:13,  2.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 500 correct 341 acc 0.6820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|███████████████████████████████████████████████████████████████▍                                                                 | 600/1221 [21:23<20:58,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 600 correct 413 acc 0.6883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████████████████████████████████████████████████████████████████████████▉                                                       | 700/1221 [25:03<19:17,  2.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 700 correct 477 acc 0.6814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|████████████████████████████████████████████████████████████████████████████████████▌                                            | 800/1221 [38:46<13:16,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 800 correct 544 acc 0.6800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████████████████████████████████████████████████████████████████████████████████████████████                                  | 900/1221 [42:24<11:14,  2.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 900 correct 619 acc 0.6878\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊                       | 1000/1221 [45:56<08:05,  2.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1000 correct 691 acc 0.6910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎            | 1100/1221 [49:29<03:36,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1100 correct 764 acc 0.6945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊  | 1200/1221 [52:56<00:45,  2.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1200 correct 831 acc 0.6925\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1221/1221 [53:40<00:00,  2.64s/it]\n"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "total = 0\n",
    "with open('outputs/dev_gpt3_complex.txt', 'w') as fd:\n",
    "    for d in tqdm(dev_data):\n",
    "        q = d['question']['stem']\n",
    "        a = d['answerKey'].lower()\n",
    "        \n",
    "        prompt_q = prompt_complex + '\\nQ: ' + q + '\\n'\n",
    "        prompt_q += 'Answer Choices:\\n'\n",
    "        for c in d['question']['choices']:\n",
    "            prompt_q += '(%c) %s\\n' % (c['label'].lower(), c['text'])\n",
    "        prompt_q += 'A:'\n",
    "\n",
    "        response = completion_with_backoff(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_q, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "\n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('Q: %s\\nA_model:\\n%s\\nA:\\n%s\\n\\n' % (q, ans_model, a))\n",
    "        \n",
    "        ans_ = ans_model.split('(')\n",
    "        if(len(ans_) > 1): ans_ = ans_[1][0]\n",
    "        else: ans_ = 'a'\n",
    "        \n",
    "        if(ans_ == a): acc += 1\n",
    "        total += 1\n",
    "        if(total % 100 == 0): \n",
    "            print('Total %d correct %d acc %.4f' % (total, acc, acc / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "6c253fda-1444-40b8-910e-fb66454b778d",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q: Laura and Bob are a married couple who have begun reaching tentative agreement regarding their sexual incompatibility. They love each other and want to remain together, but have decided that they can see other people to fulfill their needs as long as they are honest about it. Then entire negotiation is being done with a surprising amount of what given the subject matter?\n",
      "Answer Choices\n",
      "(a) calmness\n",
      "(b) compromises\n",
      "(c) fucking\n",
      "(d) anger\n",
      "(e) satisfaction\n",
      "A: The answer should be good qualities for negotiation because seeking sextual partners outside marriage is a very delicate matter.\n",
      "Of the above choices, calmness is the most suitable when discussing potential sextual partners outside marriage without breaking the merrage. \n",
      "So the answer is (a)\n",
      "\n",
      "Q: At reception desk John asked to see a person who was helping him make a sale. Lilly came out immediately and took him back to her space, and talked to him about his needs. He said that he was willing to lower the price to 80,000 if it would sale faster. Where might John be?\n",
      "Answer Choices\n",
      "(a) large office\n",
      "(b) at hotel\n",
      "(c) building\n",
      "(d) real estate office\n",
      "(e) store room\n",
      "A: The answer should be a place related to saling and also the item being saled should worth 80,000\n",
      "Of the above choices, real estate office is about selling properties and a property can be around 80,000.\n",
      "So the answer is (d)\n",
      "\n",
      "Q: Joe and Jane have begun reaching a tentative agreement about the merger of their two companies.  Their negotiations have become less than professional, and have begun doing something that is very unprofessional and compromises their objectivity. What might they be doing?\n",
      "Answer Choices\n",
      "(a) compromises\n",
      "(b) eloping\n",
      "(c) calmness\n",
      "(d) fucking\n",
      "(e) handshake\n",
      "A: The answer should be something that is not considered as normal business and influence the objectivity. \n",
      "Of the above choices, fucking is not a professional deed and would attach subjectivity to the two parties. \n",
      "So the answer is (d)\n",
      "\n",
      "Q: The building could accommodate many people.  The entrance hall alone was impressive, being wide enough to admit a hundred shoulder to shoulder.  But the building was owned by a billionaire and used only for his personal entertainment.  How would you describe this place?\n",
      "Answer Choices\n",
      "(a) convention center\n",
      "(b) public building\n",
      "(c) large building\n",
      "(d) school\n",
      "(e) town hall\n",
      "A: The answer should be somewhere that is not for the good and convenience of the people. \n",
      "Of the above choices, large building is the only neural description of a building owned by a billionaire.\n",
      "So the answer is (c)\n",
      "\n",
      "Q: Joe found that shopping  was dangerous for him.  Even though he had a great job, or maybe because of it, he found himself doing this a great deal. There were always funds left over, but not enough for his taste.  What was he doing when he was shopping?\n",
      "Answer Choices\n",
      "(a) overspending\n",
      "(b) loss of money\n",
      "(c) run out of money\n",
      "(d) spending money\n",
      "(e) deficit\n",
      "A: The answer should be something related to shopping, within the limit, and not fully satisfactory. \n",
      "Of the above choices, overspending is the choice that Joe can take but not enough. \n",
      "So the answer is (a)\n",
      "\n",
      "Q: James noticed that people who communicate well have fewer problems.  Some people communicate easily with strangers, others need to talk with members of their own groups.  A religion is a type of group.  What do members of a religion have in common?\n",
      "Answer Choices\n",
      "(a) talk to each other\n",
      "(b) believe in god\n",
      "(c) pay bills\n",
      "(d) learn from each other\n",
      "(e) believe in ghosts\n",
      "A: The answer should be something related to religion that all members share. \n",
      "Of the above choices, believing in god is a character the all members of a religion share.\n",
      "So the answer is (b)\n",
      "\n",
      "Q: John heard a language that he could not understand. He thought that the door was shut, but he eventually realized that there was no door, and that the light source that was blinding his eyes was very familiar.  He was on his back looking at what?\n",
      "Answer Choices\n",
      "(a) lamp\n",
      "(b) hallway\n",
      "(c) dard\n",
      "(d) closed room\n",
      "(e) sky\n",
      "A: The answer should be something having a blinding strong light. \n",
      "Of the above choices, the sky may have blinding light because of the sun. \n",
      "So the answer is (e)\n",
      "\n",
      "Q: What is another name for a disk for storing information?\n",
      "Answer Choices:\n",
      "(a) computer store\n",
      "(b) computer to store data\n",
      "(c) computer hard drive\n",
      "(d) cd player\n",
      "(e) usb mouse\n",
      "A:\n"
     ]
    }
   ],
   "source": [
    "print(prompt_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "7fea87fd-f91d-4067-825c-a4a14f90886d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1221 correct 847 acc 0.6937\n"
     ]
    }
   ],
   "source": [
    "print('Total %d correct %d acc %.4f' % (total, acc, acc / total))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
